package us.codecraft.demo.pipeline;

import java.util.ArrayList;
import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import us.codecraft.demo.model.IntelInfo;
import us.codecraft.demo.service.IntelInfoService;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class IntelInfoProcessor implements PageProcessor {
	private ApplicationContext context;
	
	
	
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);  
	private final static  String [] types ={"music","video","composite","commerce","joy","game","cartoon","news","star",
										"mall","pic","tour","education","sports","culture","search","food","female",
										"blog","life","gov","car","creative","fashion","art","computer","brand","magazine",
										"sucai","tools","companies","banks","mobile","sheying","waimao","daxue"};
	private static List<String> areLinks=new ArrayList<String>();
	private static List<String> urlAll=new ArrayList<String>();
	public Site getSite() {
		return site;
	}
	
	public IntelInfoProcessor() {
        context = new ClassPathXmlApplicationContext("classpath:applicationContext.xml");
    }
	
	@Autowired
    private IntelInfoService intelInfoService;
	
	public void process(Page page) {
		if(areLinks.size()<=0)
		{
			 areLinks= page.getHtml().xpath("//div[@class='container']//div[@class='filter-module']//div[1]//ul").links().all();
			 areLinks.remove(0);
		    for (String urlAre : areLinks) {
				for (String type : types) {
					urlAll.add(urlAre+type+"/");
				}
			}
		    page.addTargetRequests(urlAll);
		}else
		{
			 if(page.getHtml().xpath("//div[@class='box-body']//ul//li[1]//ul//div[@class='info pull-right']//h2//a[@class='btn-quick']").match())
			 {
				 List<String> urlTwo=page.getHtml().xpath("//div[@class='box-body']//ul//li[1]//ul//div[@class='info pull-right']//h2//a[@class='btn-quick'][1]").links().all();
				
				 page.addTargetRequests(urlTwo);
				 //分页
				 if(page.getHtml().xpath("//div[@class='box-footer']//div[@class='page-box']//a").match()&&("1").equals(page.getHtml().xpath("//div[@class='box-footer']//div[@class='page-box']//a[@class='btn-quick active']/text()")))
				 {
					 
					List<String> psgeUrl=page.getHtml().xpath("//div[@class='box-footer']//div[@class='page-box']//a").links().all();
					psgeUrl.remove(0);
					psgeUrl.remove(0);
					psgeUrl.remove(0);
					psgeUrl.remove(psgeUrl.size()-1);
					page.addTargetRequests(psgeUrl);
				 }
			 }
		    if(page.getHtml().xpath("//div[@class='website-info pull-left']").match())
		    {
		    	String title=page.getHtml().xpath("//div[@class='website-info pull-left']//ul//li[1]//h2//a/text()").toString();
		    	String type="";
		    	String url="";
		    			
		    	if(page.getHtml().xpath("//div[@class='website-info pull-left']//ul//li[4]//span/text()").match())
		    	{
		    		 type=page.getHtml().xpath("//div[@class='website-info pull-left']//ul//li[4]//span/text()").toString();
		    		 url=page.getHtml().xpath("//div[@class='website-info pull-left']//ul//li[4]//span//a/text()").toString();
		    	}else
		    	{
		    		 type=page.getHtml().xpath("//div[@class='website-info pull-left']//ul//li[3]//span/text()").toString();
		    		 url=page.getHtml().xpath("//div[@class='website-info pull-left']//ul//li[3]//span//a/text()").toString();
		    	}
		    	IntelInfo info=new IntelInfo();
		    	info.setTitle(title);
		    	info.setType(type);
		    	info.setUrl(url);
		    	System.out.println("标题："+title+"网站类型："+type+"url"+url);
		    	intelInfoService = (IntelInfoService) context.getBean("intelInfoService"); 
		        int result = intelInfoService.insert(info);
		        System.out.println(result);
		    }
		}
        System.out.println(urlAll.size()) ;
	}

	public static void main(String[] args) {
		Spider.create(new IntelInfoProcessor())  
        // 从"http://baozoumanhua.com/text"开始抓  
        .addUrl("http://www.egouz.com/asia/japan/")  
        // 开启5个线程抓取  
        .thread(10)  
        // 启动爬虫  
        .run();  
	}

}
